package cn.edu.suda.ada.worker;

import house.FiveEight;
import house.GanJi;

import java.io.IOException;
import java.util.ArrayList;
import java.util.Random;
import java.util.Scanner;
import java.util.logging.Logger;

import org.apache.http.client.ClientProtocolException;
import org.apache.http.conn.scheme.HostNameResolver;

import cn.edu.suda.ada.fetcher.PageFetcher;
import cn.edu.suda.ada.handler.ContentHandler;
import cn.edu.suda.ada.model.FetchedPage;
import cn.edu.suda.ada.model.Proxy;
import cn.edu.suda.ada.model.SpiderParams;
import cn.edu.suda.ada.parser.GanJiHouseParser;
import cn.edu.suda.ada.parser.GanJiUrlParser;
import cn.edu.suda.ada.queue.StaticUrlQueue;
import cn.edu.suda.ada.queue.UrlQueue;
import cn.edu.suda.ada.sorage.DataStorage;




public class GanJiWorker implements Runnable{
		private static final Logger Log = Logger.getLogger(GanJiWorker.class.getName());
		private static ArrayList<GanJi> ganJis=new ArrayList<GanJi>();
		
		private static UrlQueue urlQueueLevelOne=new UrlQueue();
		private UrlQueue urlQueueLevelTwo=new UrlQueue();
		
//		private PageFetcher fetcher;
//		private ContentHandler handler;
		private GanJiHouseParser info;
		private DataStorage store;
		private int threadIndex;
		private String city;
		private int proxyIndex=0;
		
		Random random = new Random();
//		private int max=12;
		private String host="ganji.com";
		
		public GanJiWorker(int threadIndex,String city){
			this.threadIndex = threadIndex;
//			this.fetcher = new PageFetcher();
//			this.handler = new ContentHandler();
			this.info=new GanJiHouseParser();
			this.store = new DataStorage(city);
			this.city=city;
			this.host=city+"."+this.host;
//			System.out.println(urlQueueLevelOne.getUrlQueue());
//			StaticUrlQueue.addElement("http://su.ganji.com/fang1/1351782251x.htm");
		}
		
		/**
		 * 设置一级页面队列
		 * @method
		 */
		public static void setUrlQueueLevelOne(UrlQueue urlQueueLevelOne) {
			GanJiWorker.urlQueueLevelOne = urlQueueLevelOne;
		}
		
		@Override
		public void run() {
			// 登录
			
			
			// 当待抓取URL队列不为空时，执行爬取任务
			// 注： 当队列内容为空时，也不爬取任务已经结束了
			//     因为有可能是UrlQueue暂时空，其他worker线程还没有将新的URL放入队列
			//	        所以，这里可以做个等待时间，再进行抓取（二次机会）
			while(!urlQueueLevelOne.isEmpty()){
				// 从待抓取队列中拿URL
				System.out.println("这是线程"+threadIndex);
				
				String url=urlQueueLevelOne.outElement();//拿出一张页面列表
				PageFetcher fetcher=new PageFetcher();
				FetchedPage page;
				
//				System.out.println("这是线程"+threadIndex);
				
				try {
//					page = fetcher.getHTML(url);
					page=fetcher.getHTMLbyProxy(url, Proxy.hostName[proxyIndex], Proxy.port[proxyIndex],host);
					
//					System.out.println("page:"+page.getContent());
					
					if(null==page.getContent()||page.getStatusCode()==403){
						System.out.println("页面状态代码："+page.getStatusCode());
						proxyIndex=random.nextInt(Proxy.proxyNum);
						Log.info("Spider-" + threadIndex + ": switch IP to "+proxyIndex);
						urlQueueLevelOne.addFirstElement(url);
						continue;
					}
					
					GanJiUrlParser urlParser=new GanJiUrlParser(city);
//					urlQueueLevelTwo.clear();
					urlQueueLevelTwo=urlParser.parse(page);
					
					int num=urlQueueLevelTwo.getUrlQueue().size();
					System.out.println("获取到URLs个数:"+num);
					System.out.println(urlQueueLevelTwo.getUrlQueue());
					int j=1;
					
					if(urlQueueLevelTwo.isEmpty()){
						proxyIndex=random.nextInt(Proxy.proxyNum);
						Log.info("Spider-" + threadIndex + ": switch IP to "+proxyIndex);
						urlQueueLevelOne.addFirstElement(url);
						continue;
					}
					
					while(!urlQueueLevelTwo.isEmpty()){
						int k=urlQueueLevelTwo.getUrlQueue().size()-1;
						System.out.println("线程"+threadIndex+"爬第"+j+"个URL，还剩"+k+"个URL");
						j++;
						
						String url2=urlQueueLevelTwo.outElement();
						PageFetcher fetcher2=new PageFetcher();
//						FetchedPage page2=fetcher2.getHTML(url2);
						FetchedPage page2=fetcher2.getHTMLbyProxy(url2, Proxy.hostName[proxyIndex], Proxy.port[proxyIndex],host);
						if(null==page.getContent()){
							proxyIndex=random.nextInt(Proxy.proxyNum);
							Log.info("Spider-" + threadIndex + ": switch IP to "+proxyIndex);
							continue;
						}
						try {
							GanJi targetData = info.parse(page2);
							System.out.println(page2.getStatusCode());
							if(null==targetData.title||""==targetData.title||"".equals(targetData.title)){
								proxyIndex=random.nextInt(Proxy.proxyNum);
								Log.info("Spider-" + threadIndex + ": switch IP to "+proxyIndex);
								continue;
							}
//							targetData.show();
							if(targetData.time.equals(targetData.crawlTime)||targetData.time.contains("08-02")
									||targetData.time.contains("08-03")||targetData.time.contains("08-04")
									||targetData.time.contains("08-05")){
								store.store(targetData,city);
							}
							
						} catch (Exception e) {
							// TODO: handle exception
							System.err.println(e);
							continue;
						}
					}
				} catch (ClientProtocolException e1) {
					// TODO Auto-generated catch block
					e1.printStackTrace();
				} catch (IOException e1) {
					// TODO Auto-generated catch block
					e1.printStackTrace();
				}
				
				// delay
				try {
					Thread.sleep(SpiderParams.DEYLAY_TIME);
				} 
				catch (InterruptedException e) {
					e.printStackTrace();
				}
			}
			
			Log.info("Spider-" + threadIndex + ": stop...");
		}
		
		public static void main(String[] args){
			for(int i = 1; i <=2; i ++){
				urlQueueLevelOne.addElement("http://"+"bj.ganji.com/fang1/o" + i);
			}
//			urlQueueLevelOne.addElement("http://bj.ganji.com/fang1/");
//			urlQueueLevelOne.addElement("http://bj.ganji.com/fang1/o2/");
			System.out.println(urlQueueLevelOne.getUrlQueue());
			for(int i=1;i<=2;i++){
//				System.out.println(urlQueueLevelOne.getUrlQueue());
				new Thread(new GanJiWorker(i,"bj")).start();
			}
//			GanJiWorker2 spiderWorker=new GanJiWorker2(1,"bj");
//			Thread thread=new Thread(spiderWorker);
//			thread.start();
		}

}
